#include "FilterData.h"
#include "SharedFileName.h"
#include "Utility.h"
#include <fstream>
#include <iostream>
#include <string>
#include <sstream>
#include <algorithm>
using namespace std;

FilterData::FilterData(string in_data_file, string stop_word_file, string in_graph_dir,string out_data_dir, bool append){
	set<string> stop_word_set;
	unordered_map<string,int> user_names_map;

	//extract sample graph information
	ExtractUserNameMap(in_graph_dir,user_names_map);
	
	// extract stop word
	ExtractStopWord(stop_word_file,stop_word_set);

	// filter data based on sample graph and stop word
	FilterDataAndOutput(in_data_file,stop_word_set, user_names_map,out_data_dir,append);
}

void FilterData::ExtractUserNameMap(string in_graph_dir,unordered_map<string,int>& user_names_map){
	string graph_file_name(in_graph_dir);
	graph_file_name.append(sample_node_file);
	//cout << "graph file: " << graph_file_name << endl;
	ifstream graphFile(graph_file_name.c_str());
	if(graphFile.is_open()){
		int ID = 0;
		string user_name;
		while(graphFile >> user_name){
			if(user_name.empty()) continue;
			user_names_map[user_name] = ID;
			ID++;
		}
		graphFile.close();
	}else{
		cout << "cannot read sample graph file." << endl;
		exit(1);
	}
}

void FilterData::ExtractStopWord(string in_stop_word_file, set<string>& stop_word_set){
	ifstream stopWordFile(in_stop_word_file.c_str());
	if(stopWordFile.is_open()){
		string stop_word;
		while(stopWordFile >> stop_word){
			// transform to lower case
			transform(stop_word.begin(), stop_word.end(), stop_word.begin(), ::tolower);
			stop_word_set.insert(stop_word);
		}
	}else{
		cout << "cannot read stop word file." << endl;
		exit(1);
	}
}

void FilterData::FilterDataAndOutput(string in_data_file, set<string>& stop_word_set,unordered_map<string,int>& user_names_map,string out_data_dir, bool append){

	ifstream inDataFile(in_data_file.c_str());
	if(!inDataFile.is_open()){
		cout << "cannot read input data file" << endl;
		exit(1);
	}
	ofstream outDataFile;
	if(append) outDataFile.open(out_data_dir.append(sample_filtered_data).c_str(),ios::out|ios::app);
	else outDataFile.open(out_data_dir.append(sample_filtered_data).c_str(),ios::out);

	int number_of_tweets = 0;
	string dummy_line; // read the first line
	getline(inDataFile,dummy_line);
	while(true){
		string publish_time;
		string author;
		string tweet;
		string empty_line;

		if(dummy_line.at(1) == '\t'){
			publish_time = dummy_line;
			dummy_line = "NOT USABLE";
		}else{
			getline(inDataFile,publish_time);
			if(publish_time.empty()) break;
		}
		publish_time.erase(0,2); // erase first two characters
		
		getline(inDataFile,author);
		if(author.empty()) break;
		author.erase(0,21);      // erase until the name of the users
		
		getline(inDataFile,tweet);
		if(tweet.empty()) break;
		tweet.erase(0,2);

		getline(inDataFile,empty_line); // read the empty line
		unordered_map<string,int>::iterator it = user_names_map.find(author);
		if(it == user_names_map.end()) continue; // the tweet is not published by sample users

		string tweet_origin(tweet);
		if(IsStringContainNonAsciiChar(tweet)) continue; // if the tweet contains non-ascii code, skip
		tweet = ReplaceStrangeCharacterWithSpace(tweet);
		if(tweet.compare("no post title") == 0) continue;

		stringstream iss(tweet);
		stringstream oss;
		string token;
		while(iss >> token){
			if(IsTokenKeyword(token,stop_word_set))
				oss << token << " ";
		}
		tweet = oss.str();
		if(tweet.empty()) continue;

		outDataFile << publish_time << endl;
		outDataFile << it->second << endl; // change to user ID
		outDataFile << tweet << endl;


		number_of_tweets++;
		if(number_of_tweets % 1000000 == 0) cout << "collected " << (number_of_tweets / 1000000) << "M tweets" << endl;
	}

	inDataFile.close();
	outDataFile.close();
}

int main(int argc, char *argv[]){
	if(argc != 6){
		cout << "wrong arguments for " << argv[0] << endl;
		cout << "1. input tweets data you want to filter" << endl;
		cout << "2. file path for stop words" << endl;
		cout << "3. input directory for sampled data" << endl;
		cout << "4. output directory for the filtered data" << endl;
		cout << "5. 0 will rewrite the existing file, 1 will append the existing file" << endl;
		exit(1);
	}

	string in_data_file(argv[1]);
	string stop_word_file(argv[2]);
	string in_graph_dir(argv[3]);
	string out_graph_dir(argv[4]);
	bool append = (bool)atoi(argv[5]);

	FilterData filter(in_data_file,stop_word_file,in_graph_dir,out_graph_dir,append);
	//FilterData filter("./data/tweets/tweets2009-07.txt","./data/stop_words.txt","./sampledData","./sampledData",true);

	cout << "finished filtering data" << endl;
}